#将文本及音标行数据格式化，比如文本行：序号+\t+文本(文本间以空格隔开);音标行\t+音标(以空格隔开)
#去除除了指定符号外的其他字符
#,.?;!~%/前面不能有空格,后面必须有空格(除句末)
#-前面不能有空格

import re,os,sys
def read_file(filename):
    try:
        try:
            try:
                with open(filename,'r',encoding='utf-8') as r:
                    lines=r.readlines()
            except:
                with open(filename, 'r', encoding='gbk') as r:
                    lines = r.readlines()
        except:
            with open(filename, 'r', encoding='utf-16') as r:
                lines = r.readlines()
    except:
        with open(filename, 'r', encoding='GB2312') as r:
            lines = r.readlines()
    return lines
def loaddic():
    dic={}
    lines= read_file('勿删/合法音素词典.txt')
    for line in lines:
        line=line.strip()
        if line not in dic:
            dic[line]='0'
    return dic
#对文本行数据进行格式化
def get_newstr(str):
    subsentence=str.replace('，',',').replace('。','.').replace('’','\'').replace('？','?')
    subsentence=subsentence.replace('；',';').replace('！','!').replace('~','~').replace('%','%').replace('/','/')
    subsentence=re.sub('(\s+)?([^\.,/\w\s\-\' ́])(\s+)?','\\2 ',subsentence) #将符号(除了-')前面的空格都替换掉，后面的空格都保留
    # subsentence =re.sub('\s+(-)','\\1',subsentence)  # 单独处理-,替换-前面的空格
    subsentence = re.sub('[^\w,.?;!\-~%/\'`–]', ' ', subsentence)  # 保留符号,.?;!-'，其他都去掉
    subsentence = re.sub(r'\s+([^\w\'])', '\\1', subsentence)  # 替换标点符号前面的空格
    subsentence = re.sub('\s+', ' ', subsentence)  # 保留符号,.?;!-，其他都去掉
    return subsentence.strip()
#对音标行进行格式化
def get_new_rhstr(rh_sentence,dic,notindic,newnoindic):
    rh_sentence=re.sub('[^\w\s/.~ː:`]',' ',rh_sentence) #音标行替换除了\w空格/.~以外的其他字符
    new_rh_word = []
    rh_word_list = re.split('/', rh_sentence)
    for rh_word in rh_word_list:
        rh_word = rh_word.strip()
        if len(rh_word) > 0:
            # 音节
            new_ele_list = []
            ele_list = re.split('\.', rh_word)
            for ele in ele_list:
                ele = ele.strip()
                if len(ele) > 0:
                    split_ele = re.split('\s+', ele)
                    new_ele = []
                    for e in split_ele:
                        e = e.strip()
                        if len(e) == 1:
                            if e not in dic:
                                if e not in notindic:
                                    notindic[e] = e
                            new_ele.append(e)
                        elif len(e) == 2:
                            if e[-1] in ['1','2']:
                                new_ele.append(e)
                                if e[0] not in dic and e[0] not in notindic:
                                    notindic[e[0]] = e
                            elif e[-1].isdigit():
                                for c in e:
                                    new_ele.append(c)
                            elif e in dic:
                                new_ele.append(e)
                            else:
                                if e not in notindic:
                                    notindic[e] = e
                                for c in e:
                                    new_ele.append(c)
                        else:
                            j = 0
                            while j < len(e):
                                if e[j:j + 2] in dic:
                                    new_ele.append(e[j:j + 2])
                                    j += 2
                                    continue
                                elif e[j:j + 1] in dic:
                                    new_ele.append(e[j])
                                    j += 1
                                    continue
                                else:
                                    if e[j] not in notindic:
                                        notindic[e[j]] = e
                                    new_ele.append(e[j])
                                    j += 1
                                    continue
                    new_ele1 = re.sub('(\s+)(\d)', '\\2', ' '.join(new_ele)) #去掉数字前面的空格
                    new_ele1 = re.sub('(\d)+', '\\1', new_ele1) #多个数字用一个数字代替
                    for ele in new_ele1.split(' '):
                        ele = ele.strip()
                        if ele[-1] in ['1','2']:
                            ele = ele[:-1]
                        if ele not in dic and ele not in newnoindic:
                            newnoindic[ele] = rh_word
                    new_ele_list.append(new_ele1)
            new_word = ' . '.join(new_ele_list)
            new_rh_word.append(new_word)
    return new_rh_word,notindic,newnoindic
def get_newline(filename,out_file,errodic):
    _,file=os.path.split(filename)
    try:
        lines=read_file(filename)
        newsentence_list=[]
        newrhsentence_list=[]
        dic=loaddic()
        dic=sorted(dic.items(),key=lambda x:len(x[0]),reverse=True)
        newdic=[i[0] for i in dic]
        notindic={} #没修改之前的不在字典中的音素
        newnoindic={} #修改后的不在字典中的音素
        if len(lines)%2==0:
            for i in range(0,len(lines),2):
                subsentence = lines[i].strip() #德语不格式化符号类型
                # subsentence = get_newstr(lines[i].strip())  # 格式标准化
                split_subsentence = re.split('\s+', subsentence)  # 以空格分开
                sequnece = split_subsentence[0]  # 序号
                newsentence = ' '.join(split_subsentence[1:])  # 以空白符分割开后以空格连接，文本
                rh_sentence=lines[i+1]
                if not sequnece.isdigit():
                    if file not in errodic:
                        errodic[file]=['第{}行数据序号错误'.format(i+1)]
                    else:
                        errodic[file].append('第{}行数据序号错误'.format(i+1))
                newsentence = sequnece+'\t'+newsentence
                if re.findall('\W',newsentence[-1]):
                    newsentence=newsentence[:-1].strip()+newsentence[-1]
                # result=re.findall('[\u00C0-\u01FFa-zA-Z]+\s+[^"\u00C0-\u01FFa-zA-Z]+',newsentence)
                # if result!=[]:
                #     newsentence=re.sub('([\u00C0-\u01FFa-zA-Z0-9]+)\s+([^"\u00C0-\u01FFa-zA-Z0-9]+)','\\1\\2',newsentence) #word -替换为word-
                tempsentencec=newsentence.split(' ')
                newsentence=' '.join(tempsentencec[:-1])+' '+re.sub('(%)(.*?)$','\\1.',tempsentencec[-1])
                newsentence_list.append(newsentence)
                new_rh_word,notindic,newnoindic=get_new_rhstr(rh_sentence, newdic,notindic,newnoindic)
                newrhsentence_list.append('\t'+' / '.join(new_rh_word)+'\n')
            with open(out_file,'w',encoding='utf-8') as w:
                for i in range(len(newsentence_list)):
                    w.write(newsentence_list[i]+'\n'+newrhsentence_list[i])
        else:
            if file not in errodic:
                errodic[file]=['文本行数不为偶数，一行文本一行音标']
            else:
                errodic[file].append('文本行数不为偶数，一行文本一行音标')
    except Exception as e:
        if file not in errodic:
            errodic[file]=[str(e)]
        else:
            errodic[file].append(str(e))
    return errodic
def formatstr(in_path,out_path):
    mkdir(out_path)
    errodic={}#记录每个文件中的错误信息
    for rt, _, files in os.walk(in_path):
        for file in files:
            if file.endswith('.txt'):
                filename = os.path.join(rt, file)
                newfile=os.path.join(out_path,file)#输出文件
                errodic=get_newline(filename, newfile,errodic)
    if errodic!={}:
        with open('文本格式化错误.txt','w',encoding='utf-8') as w:
            for k,values in errodic.items():
                w.write('*'*20+k+'*'*20+'\n'+'\n'.join(values)+'\n')
def mkdir(dst):
    if not os.path.join(dst):
        os.makedirs(dst)
def record(out_path):
    recorddic={}#记录词频
    record_erro={}#记录错误
    for rt,_,files in os.walk(out_path):
        for file in files:
            try:
                filename=os.path.join(rt,file)
                lines=read_file(filename)
                for i in range(0,len(lines),2):
                    splitline=lines[i].strip().split('\t')
                    sequence=splitline[0]
                    sentence=re.sub('[^a-z\u00C0-\u00FF\s]','',splitline[1].lower())
                    wordlist=re.split('\s+',sentence)
                    poneticlist=re.split(' / ',lines[i+1].strip())
                    if len(wordlist)==len(poneticlist):
                        for j in range(len(wordlist)):
                            kword=wordlist[j]+'\t'+poneticlist[j]
                            if kword not in recorddic:
                                recorddic[kword]=1
                            else:
                                recorddic[kword] += 1
                    else:
                        if file not in record_erro:
                            record_erro[file]=['{}\t文本与音标长度不一致\n{}\n{}'.format(sequence,str(len(wordlist))+' '+str(wordlist),str(len(poneticlist))+' '+str(poneticlist))]
                        else:
                            record_erro[file].append('{}\t文本与音标长度不一致\n{}\n{}'.format(sequence,str(len(wordlist))+' '+str(wordlist),str(len(poneticlist))+' '+str(poneticlist)))
            except Exception as e:
                if file not in record_erro:
                    record_erro[file]=[str(e)]
                else:
                    record_erro[file].append(str(e))
    recorddic1={}
    for k, v in recorddic.items():
        word=k.split('\t')[0]
        phonetic=k.split('\t')[1]
        if word not in recorddic1:
            recorddic1[word]=[phonetic+'\t'+str(v)]
        else:
            recorddic1[word].append(phonetic+'\t'+str(v))
    if recorddic1!={}:
        recordfile = '词频记录.txt'
        with open(recordfile, 'w', encoding='utf-8') as w:
            for k,values in recorddic1.items():
                w.write(k+'\t'+'\t'.join(values)+'\n')
    if record_erro!={}:
        recorderrofile = '词频统计错误.txt'
        with open(recorderrofile, 'w', encoding='utf-8') as w:
            for k,values in record_erro.items():
                w.write(k+'\n'+'\n'.join(values)+'\n')
    return recorddic
def main():
    if len(sys.argv) < 3:
        print(sys.stderr, "usage: python 德语格式化处理.py in_path out_path")
        exit()
    else:
        # filename=r'2-7-8.txt'
        # out_file = '11.txt'
        path = sys.argv[1]
        output = sys.argv[2]
        if not os.path.exists(output):
            os.makedirs(output)
        formatstr(path, output)
        record(output)
    # path = input('输入路径：')
    # output = input('输出路径：')
    # formatstr(path,output)
    # record(output)
main()